{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "本代码主要作用是 整理CAK/WL目录下文件列表的数据统计，因为涉及到爬虫，所以处理速度会比较慢，视网速而定。如需单日统计数据，请移步至CAK-WL 生成单日数据.ipynb 文件处理，维护QQ：471238751\n",
    "使用须知：代码自动生成CAK/WL/HA的统计表格，请确保目录下不存在‘文件整理表格.xls’文件，否则报错，删除掉文件即可运行\n",
    "1次只能生成1个通道数据\n",
    "下面正式开始\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "执行完毕\n",
      "cak\n",
      "flat\n",
      "ha\n",
      "wl\n",
      "\\\\10.153.110.37\\Data\\solar_observation\\2021\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import os,datetime\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import paramiko\n",
    "\n",
    "# 建立一个sshclient对象\n",
    "ssh = paramiko.SSHClient()\n",
    "# 允许将信任的主机自动加入到host_allow 列表，此方法必须放在connect方法的前面\n",
    "ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "# 调用connect方法连接服务器\n",
    "# 需要管理员账号\n",
    "ssh.connect(hostname='10.153.110.37', port=22, username='admin', password='admin')\n",
    "# 执行命令\n",
    "# 关机命令\n",
    "#command=\"shutdown -h now\"\n",
    "#stdin, stdout, stderr = ssh.exec_command(command)\n",
    "# 结果放到stdout中，如果有错误将放到stderr中\n",
    "#print(stdout.read().decode())\n",
    "print('执行完毕')\n",
    "os.chdir(r'\\\\10.153.110.37\\Data\\solar_observation\\2021')\n",
    "for i in os.scandir():\n",
    "    print(i.name)\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用须知：代码自动生成CAK/WL/HA的统计表格，请确保目录下不存在‘文件整理表格.xls’文件，否则报错，删除掉文件即可运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#如果观测时间分上下午，上午时间一个列表，下午时间一个列表，分开求和再相加得出总的时间\n",
    "def 文件夹日期(path):\n",
    "    return re.search('\\d+',path).group()\n",
    "\n",
    "def 时间计算(time):\n",
    "    if time != []:\n",
    "        a = time[-1]\n",
    "        b = time[0]\n",
    "        c = round((a-b).seconds/60/60,2)\n",
    "        return round(c,2)\n",
    "    else:\n",
    "        return 0\n",
    "def 时间段(time):\n",
    "    if time != []:\n",
    "        b=time[0]\n",
    "        c=time[-1]\n",
    "        return '-'.join((b.strftime('%X'),c.strftime('%X')))\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "    \n",
    "#爬取太阳黑子，网站https://www.solarmonitor.org/index.php?date=20201023&region=&indexnum=1\n",
    "def 黑子(num):\n",
    "    res = requests.get('https://www.solarmonitor.org/index.php?date='+str(num))\n",
    "    bs = BeautifulSoup(res.text,'html.parser')\n",
    "    noaa=bs.find_all('tr',class_='noaaresults')\n",
    "    if noaa ==[]:\n",
    "        #编号\n",
    "        编号.append('平静')\n",
    "        #方位\n",
    "        方位.append('平静')\n",
    "        #黑子数量\n",
    "        黑子数量.append('无黑子')\n",
    "        #positions = position.replace('S','+').replace('N','-')   #需求要把南北换成 +\\-\n",
    "        #耀斑爆发等级和时间\n",
    "        耀斑爆发等级和时间.append('平静')\n",
    "    else:\n",
    "        a=[]\n",
    "        b=[]\n",
    "        c=[]\n",
    "        d=[]\n",
    "        for s in noaa:\n",
    "            #编号\n",
    "            a.append(s.find(id='noaa_number').text)\n",
    "            #方位\n",
    "            b.append(s.find(id='position').text.replace(' ',''))\n",
    "            #黑子数量\n",
    "            c.append(s.find(id='nspots').text.replace(' ',''))\n",
    "            #positions = position.replace('S','+').replace('N','-')   #需求要把南北换成 +\\-\n",
    "            #耀斑爆发等级和时间\n",
    "            d.append(s.find(id='events').text.replace(' ',''))\n",
    "        编号.append(a)\n",
    "        方位.append(b)\n",
    "        黑子数量.append(c)\n",
    "        耀斑爆发等级和时间.append(d)\n",
    "\n",
    "#表格抬头 Z是字典\n",
    "\n",
    "def all(num):\n",
    "    #os.chdir(r'E:\\shuju\\num')\n",
    "    os.chdir(os.path.join(ppath,num))\n",
    "    mtime = [] #存放上午时间段\n",
    "    ntime = []#存放下午时间段\n",
    "    size = 0 #文件大小\n",
    "    lensize = 0#文件数量\n",
    "    for i in os.scandir(): #单个文件夹资料\n",
    "        if i.name.endswith('.fit'):\n",
    "            #文件大小\n",
    "            size += os.path.getsize(i)/1024/1024\n",
    "            #文件数量\n",
    "            lensize +=1\n",
    "            #print(str('%.2f' %size)+'M')\n",
    "            #print(os.path.getctime(i))\n",
    "            #dt.datetime.fromtimestamp(os.path.getmtime(i).append(mtime)\n",
    "\n",
    "            #创建时间循环\n",
    "            mmtime=dt.datetime.fromtimestamp(os.path.getmtime(i))\n",
    "            if mmtime.hour < 13:   #小于13时的时间放入mmtime列表内\n",
    "                mtime.append(mmtime)\n",
    "            elif 13 < mmtime.hour < 19: #大于13时小于19时的时间放入nmtime列表内\n",
    "                ntime.append(mmtime)\n",
    "    #输出需要的资料，并打包\n",
    "    #print(os.path.split(os.getcwd())[1][:-3])\n",
    "    日期.append(文件夹日期(os.path.split(os.getcwd())[1]))\n",
    "    #print(时间段(mtime))\n",
    "    观测时间段1.append(时间段(mtime))\n",
    "    #print(时间段(ntime))\n",
    "    观测时间段2.append(时间段(ntime))\n",
    "    #print(时间计算(mtime)+时间计算(ntime))\n",
    "    总观测时间.append(时间计算(mtime)+时间计算(ntime))\n",
    "    #print(lensize)\n",
    "    观测文件.append(lensize)\n",
    "    #print('文件总大小'+str('%.2f' %size)+'M')\n",
    "    文件总大小.append(str('%.2f' %size)+'M')\n",
    "    黑子(re.search('\\d+',num).group())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#在ppath填入CAK/WL/HA路径\n",
    "ppath=r'\\\\10.153.110.37\\Data\\solar_observation\\2021\\wl'\n",
    "os.chdir(ppath)\n",
    "\n",
    "fname=[]\n",
    "日期=[]\n",
    "观测时间段1=[]\n",
    "观测时间段2=[]\n",
    "总观测时间=[]\n",
    "观测文件=[]\n",
    "文件总大小=[]\n",
    "编号=[]\n",
    "方位=[]\n",
    "黑子数量=[]\n",
    "耀斑爆发等级和时间=[]\n",
    "z={'日期':日期,'观测时间段1':观测时间段1,'观测时间段2':观测时间段2,'总观测时间':总观测时间,'观测文件':观测文件,'文件总大小':文件总大小,'编号':编号,'方位':方位,'黑子数量':黑子数量,'耀斑爆发等级和时间':耀斑爆发等级和时间}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['20210101WL', '20210102WL', '20210103WL', '20210105WL', '20210106WL', '20210107WL', '20210109WL', '20210112WL', '20210113WL', '20210114WL', '20210115WL', '20210116WL', '20210117WL', '20210118WL', '20210119WL', '20210120WL', '20210122WL', '20210123WL', '20210124WL', '20210125WL', '20210126WL', '20210127WL', '20210128WL', '20210129WL', '20210130WL', '20210131WL', '20210201WL', '20210202WL', '20210203WL', '20210204WL', '20210205WL', '20210206WL']\n",
      "<bound method NDFrame.head of                      观测时间段1             观测时间段2  总观测时间  观测文件     文件总大小  \\\n",
      "日期                                                                      \n",
      "20210101  09:19:27-11:56:03  14:29:13-17:38:11   5.76   327  5233.42M   \n",
      "20210102  09:31:51-12:01:17  14:00:00-16:29:32   4.98   268  4289.16M   \n",
      "20210103  10:55:54-12:01:35  15:01:15-17:26:45   3.51   192  3072.83M   \n",
      "20210105               None               None   0.00     0     0.00M   \n",
      "20210106               None  14:34:21-17:23:03   2.81   153  2448.66M   \n",
      "20210107  10:54:12-11:59:53               None   1.09    66  1056.29M   \n",
      "20210109  09:05:58-11:43:27  14:23:02-15:16:35   3.51   292  4673.27M   \n",
      "20210112  09:11:16-12:00:00  14:14:20-17:45:32   6.33   346  5537.50M   \n",
      "20210113  09:10:26-12:00:15  14:19:59-17:45:11   6.25   349  5585.51M   \n",
      "20210114  09:09:12-12:00:37  14:20:49-17:35:49   6.11   322  5153.40M   \n",
      "20210115  09:06:37-12:00:23  14:22:31-17:26:25   5.96   319  5105.38M   \n",
      "20210116  09:25:24-12:01:00  14:30:31-15:53:21   3.97   229  3664.99M   \n",
      "20210117  09:18:54-12:01:25  14:28:55-18:01:06   6.25   337  5393.46M   \n",
      "20210118  09:44:44-12:03:16  14:51:53-17:39:44   5.11   291  4657.26M   \n",
      "20210119  09:29:01-12:06:44               None   2.63   150  2400.65M   \n",
      "20210120               None  16:37:42-17:19:07   0.69    39   624.17M   \n",
      "20210122  09:25:10-11:59:47  14:26:17-17:28:07   5.61   279  4465.21M   \n",
      "20210123  09:17:43-12:00:21  14:23:56-17:30:54   5.83   327  5233.42M   \n",
      "20210124  09:32:55-09:53:08  14:19:14-17:31:15   3.54   192  3072.83M   \n",
      "20210125  09:42:52-12:05:20  14:30:08-17:55:24   5.79   339  5425.47M   \n",
      "20210126  09:20:20-12:03:00  14:39:44-17:37:35   5.67   304  4865.32M   \n",
      "20210127  09:37:45-11:58:20  14:34:45-17:28:31   5.24   186  2976.81M   \n",
      "20210128  09:27:45-11:57:54  14:38:03-17:37:54   5.50   184  2944.80M   \n",
      "20210129  09:42:34-10:18:58  14:34:38-15:55:30   1.96   303  4849.31M   \n",
      "20210130  09:29:56-10:58:50  14:22:42-16:51:12   3.96   342  5473.48M   \n",
      "20210131  09:11:17-11:50:57  14:21:30-16:43:54   5.03   349  5585.51M   \n",
      "20210201  09:44:52-10:57:41  14:35:26-15:26:58   2.07   335  5361.45M   \n",
      "20210202  09:18:28-12:04:07  14:28:48-17:41:48   5.98   284  4545.23M   \n",
      "20210203  09:25:10-09:50:25  14:50:15-14:45:12  24.34   302  4833.31M   \n",
      "20210204  09:34:49-12:11:27  14:22:41-17:29:34   5.72   323  5169.40M   \n",
      "20210205  09:02:18-11:55:05  14:23:13-17:46:20   6.26   335  5361.45M   \n",
      "20210206  09:03:01-11:59:52  14:21:32-17:54:40   6.50   351  5617.52M   \n",
      "\n",
      "                                        编号  \\\n",
      "日期                                           \n",
      "20210101                  [ 12794,  12795]   \n",
      "20210102                  [ 12794,  12795]   \n",
      "20210103                  [ 12794,  12795]   \n",
      "20210105                          [ 12795]   \n",
      "20210106                                平静   \n",
      "20210107                                平静   \n",
      "20210109                                平静   \n",
      "20210112                                平静   \n",
      "20210113                                平静   \n",
      "20210114                                平静   \n",
      "20210115                                平静   \n",
      "20210116                          [ 12796]   \n",
      "20210117                          [ 12796]   \n",
      "20210118                  [ 12796,  12797]   \n",
      "20210119                  [ 12797,  12796]   \n",
      "20210120                  [ 12797,  12796]   \n",
      "20210122          [ 12797,  12798,  12796]   \n",
      "20210123          [ 12797,  12798,  12799]   \n",
      "20210124          [ 12797,  12798,  12799]   \n",
      "20210125          [ 12797,  12799,  12798]   \n",
      "20210126          [ 12797,  12799,  12798]   \n",
      "20210127          [ 12797,  12799,  12798]   \n",
      "20210128  [ 12797,  12800,  12798,  12799]   \n",
      "20210129          [ 12797,  12798,  12800]   \n",
      "20210130          [ 12797,  12798,  12800]   \n",
      "20210131                          [ 12800]   \n",
      "20210201                          [ 12800]   \n",
      "20210202                          [ 12800]   \n",
      "20210203                          [ 12801]   \n",
      "20210204                          [ 12801]   \n",
      "20210205                          [ 12801]   \n",
      "20210206                          [ 12801]   \n",
      "\n",
      "                                                         方位  \\\n",
      "日期                                                            \n",
      "20210101           [S16W80(924\",-259\"), S18W50(712\",-269\")]   \n",
      "20210102           [S16W91(937\",-269\"), S17W63(832\",-261\")]   \n",
      "20210103           [S16W91(937\",-269\"), S18W80(914\",-291\")]   \n",
      "20210105                               [S18W91(927\",-301\")]   \n",
      "20210106                                                 平静   \n",
      "20210107                                                 平静   \n",
      "20210109                                                 平静   \n",
      "20210112                                                 平静   \n",
      "20210113                                                 平静   \n",
      "20210114                                                 平静   \n",
      "20210115                                                 平静   \n",
      "20210116                               [S21W19(297\",-278\")]   \n",
      "20210117                               [S21W32(484\",-284\")]   \n",
      "20210118          [S21W44(634\",-292\"), S18E64(-834\",-265\")]   \n",
      "20210119          [S17E55(-765\",-237\"), S21W58(773\",-306\")]   \n",
      "20210120          [S17E43(-637\",-223\"), S21W72(866\",-323\")]   \n",
      "20210122  [S18E10(-161\",-216\"), S16E22(-352\",-188\"), S21...   \n",
      "20210123  [S18W03(48\",-213\"), S16E06(-98\",-180\"), N21W42...   \n",
      "20210124  [S17W16(257\",-199\"), S16W07(114\",-179\"), N22W5...   \n",
      "20210125  [S17W28(439\",-204\"), N23W71(848\",407\"), S16W21...   \n",
      "20210126  [S18W41(609\",-231\"), N22W82(894\",375\"), S16W35...   \n",
      "20210127  [S17W54(755\",-229\"), N23W91(896\",376\"), S16W49...   \n",
      "20210128  [S18W63(826\",-257\"), N19W27(419\",399\"), S16W58...   \n",
      "20210129  [S18W81(915\",-284\"), S16W77(912\",-245\"), N18W4...   \n",
      "20210130  [S18W91(925\",-300\"), S16W90(935\",-266\"), N18W6...   \n",
      "20210131                                [N18W62(818\",344\")]   \n",
      "20210201                                [N18W86(923\",305\")]   \n",
      "20210202                                [N18W91(925\",297\")]   \n",
      "20210203                                [N31W55(684\",551\")]   \n",
      "20210204                                [N30W71(797\",514\")]   \n",
      "20210205                                [N30W84(837\",493\")]   \n",
      "20210206                                [N30W91(841\",481\")]   \n",
      "\n",
      "                            黑子数量             耀斑爆发等级和时间  \n",
      "日期                                                      \n",
      "20210101          [01/01, 04/05]                [-, -]  \n",
      "20210102          [01/01, 02/04]                [-, -]  \n",
      "20210103          [01/01, 01/02]                [-, -]  \n",
      "20210105                     [/]                   [-]  \n",
      "20210106                     无黑子                    平静  \n",
      "20210107                     无黑子                    平静  \n",
      "20210109                     无黑子                    平静  \n",
      "20210112                     无黑子                    平静  \n",
      "20210113                     无黑子                    平静  \n",
      "20210114                     无黑子                    平静  \n",
      "20210115                     无黑子                    平静  \n",
      "20210116                 [03/--]                   [-]  \n",
      "20210117                 [05/03]                   [-]  \n",
      "20210118          [01/05, 02/--]                [-, -]  \n",
      "20210119            [03/02, /01]  [C1.0(17:46)\\n/-, -]  \n",
      "20210120              [04/03, /]  [-/C1.0(17:46)\\n, -]  \n",
      "20210122       [02/02, 04/03, /]             [-, -, -]  \n",
      "20210123   [02/02, 05/04, 02/--]             [-, -, -]  \n",
      "20210124   [02/02, 01/05, 01/02]             [-, -, -]  \n",
      "20210125     [02/02, 01/01, /01]             [-, -, -]  \n",
      "20210126       [02/02, 04/01, /]             [-, -, -]  \n",
      "20210127       [02/02, 01/04, /]             [-, -, -]  \n",
      "20210128  [02/02, 04/--, /, /01]          [-, -, -, -]  \n",
      "20210129           [/02, /, /04]             [-, -, -]  \n",
      "20210130               [/, /, /]             [-, -, -]  \n",
      "20210131                     [/]                   [-]  \n",
      "20210201                     [/]                   [-]  \n",
      "20210202                     [/]                   [-]  \n",
      "20210203                 [02/--]                   [-]  \n",
      "20210204                 [01/02]                   [-]  \n",
      "20210205                   [/01]                   [-]  \n",
      "20210206                     [/]                   [-]  >\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "for f in os.scandir():  #遍历目录\n",
    "    if f.is_dir: #如果发现文件夹\n",
    "        fname.append(f.name)\n",
    "fname.sort()\n",
    "print(fname)\n",
    "\n",
    "\n",
    "for i in fname:\n",
    "    all(i)\n",
    "df = pd.DataFrame.from_dict(z)\n",
    "df.set_index('日期',inplace = True)\n",
    "print(df.head)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#查看字典Z的数据，大致表格情况\n",
    "print(len(z['编号']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(ppath)\n",
    "df.to_excel(ppath+'\\文件整理表格.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
